﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices.ComTypes;
using System.Text;
using System.Text.RegularExpressions;
using System.Data.SqlClient;
using System.Configuration;
using System.Data;
using System.IO;



using System.Windows.Forms;

namespace TMScore_Subsets
{
    class Parser
    {
        private static Hashtable stopWordsHash; // hastable of stopwords read from stopwords.txt
        private static bool useStemmer;
        public static Stemmer stemmer;
        private static Dictionary<string, string> m_AllDocs; // the large term dictionary that we will use in Form1 for outputing results

        public static Dictionary<string, string> MainFunction(bool parse, Dictionary<string, string> m_AllDocs2)
        {
            stemmer = new Stemmer();
            string text;
            m_AllDocs = new Dictionary<string, string>();
            stopWordsHash = new Hashtable();
            Stopwords(); // Read the stopwords from stopwords.text
            useStemmer = true;
            foreach (KeyValuePair <string, string> pair in  m_AllDocs2)
            {

                text = Process_Text(pair.Value);
                m_AllDocs[pair.Key]=text;
            }
            return m_AllDocs;
        }

        public static string Process_Text(string document_text)
        {
            //change everything to lower
            document_text = document_text.ToLower();

            //remove all the stuff added by news agencies
            document_text = Remove_IPTC_Prefix_Suffix(document_text);

            //document_text = Remove_Stop_Words(document_text);

            //get rid of problematic characters
            document_text = Remove_Problematic_Chars(document_text);

            //get rid of "new line" characters
            document_text = document_text.Replace("\r", "");
            document_text = document_text.Replace("\n", " ");
            document_text = document_text.Replace("\t", " ");

            //getting rid of double spaces
            while (document_text.Contains("  "))
            {
                document_text = document_text.Replace("  ", " ");
            }

            //getting rid of problematic chars
            document_text = document_text.Replace("~", "");
            document_text = document_text.Replace(";", "");

            string[] split_text = document_text.Split(' ');

            //get rid of all the stop words
            for (int i = 0; i < split_text.Length; i++)
            {
                if (stopWordsHash.Contains(split_text[i])  || split_text[i].Length > 150)
                {
                    split_text[i] = "";
                }
            }

            //stem each remaining word
            for (int i = 0; i < split_text.Length; i++)
            {
                if (split_text[i] != "")
                {
                    //split_text[i] = Stemmers.Lovins_Stemmer.Stem(split_text[i]);
                    stemmer.add(split_text[i].ToCharArray(), split_text[i].Length); // use the stemmer
                    stemmer.stem();
                    split_text[i] = stemmer.ToString();
                }
            }

            StringBuilder text_to_return = new StringBuilder();

            //join the array back to a single string
            foreach (string word in split_text)
            {
                if (word != "" && !Contains_Only_Numbers(word))
                {
                    //text_to_return = String.Concat(text_to_return, " ", word);
                    text_to_return.Append(word);
                    text_to_return.Append(" ");
                }
                else
                {
                    int x = 5;
                }
            }

            return text_to_return.ToString().Trim();
        }

        private static string Remove_IPTC_Prefix_Suffix(string document_text)
        {
            List<string> possible_prefixes = new List<string>() { "(reuters)" };
            List<string> possible_suffixes = new List<string>() { "(writing by", "(editing by", "(reporting by" };

            foreach (string prefix in possible_prefixes)
            {
                int pos = document_text.IndexOf(prefix);
                if (pos != -1 && pos < 300)
                {
                    document_text = document_text.Substring(pos + prefix.Length);
                }
            }

            foreach (string suffix in possible_suffixes)
            {
                int pos = document_text.IndexOf(suffix);
                if (pos != -1)
                {
                    document_text = document_text.Substring(0, pos);
                }
            }

            return document_text;
        }

        private static bool Contains_Only_Numbers(string word)
        {
            Regex objNotNaturalPattern = new Regex(@"^[0-9]+$");
            return objNotNaturalPattern.IsMatch(word);
        }

        public static string Remove_Problematic_Chars(string text)
        {
            text = Remove_Tags(text);

            //this does not contain the "." and ":" for a good reason - this interferes 
            //with the scores given to the confidential terms
            List<string> chars_to_remove = new List<string>() { ",", "(", ")", "-", "_", "[", "]", "{", "}", "#", "*", "\"",
                "+", "!","|",  @"//", @"////", @"///", @"/", @"\", "&", "=", "?", "'", "http", "www", "<<<", ">>>", "mergeformat", "<", ">",                
            };

            foreach (string char_to_remove in chars_to_remove)
            {
                text = text.Replace(char_to_remove, " ");
            }

            string[] split_text = text.Split(' ');
            StringBuilder sb = new StringBuilder();

            bool in_confidential_tag = false;
            foreach (string word in split_text)
            {
                if (word == "")
                {
                    continue;
                }
                if (!in_confidential_tag && !word.Contains("<confidential:"))
                {
                    string cleaned_word = word.Replace(".", " ").Replace(":", " ");
                    if (cleaned_word.Contains("<") || cleaned_word.Contains(">"))
                    {
                        cleaned_word = cleaned_word.Replace("<", " ").Replace(">", " ");
                    }

                    sb.Append(cleaned_word);
                }
                else
                {
                    in_confidential_tag = true;
                    sb.Append(word);
                }
                if (word.Contains("</confidential"))
                {
                    in_confidential_tag = false;
                }
                sb.Append(" ");
            }
            string new_text = sb.ToString();
            while (new_text.Contains("  "))
            {
                new_text = new_text.Replace("  ", " ");
            }

            return new_text;
        }

        private static string Remove_Tags(string text)
        {
            StringBuilder sb = new StringBuilder();
            while (text.Contains("<"))
            {
                int start_pos = text.IndexOf("<");
                sb.Append(text.Substring(0, start_pos) + " ");
                text = text.Substring(start_pos);
                int end_pos = text.IndexOf(">");
                if (end_pos != -1)
                {
                    text = text.Substring(end_pos + 1);
                }
                else
                {
                    text = text.Substring(1);
                }
            }
            sb.Append(text);
            return sb.ToString();
        }

        public static void Stopwords()
        {
            StreamReader inputFile = File.OpenText("stopWords.txt");
            string stopword = inputFile.ReadLine();
            stopword = stopword.ToLower();
            while (!inputFile.EndOfStream)
            {
                if (!stopWordsHash.Contains(stopword))
                {
                    stopWordsHash.Add(stopword, stopword); // Using hastable for quick search
                }
                stopword = inputFile.ReadLine();
                stopword = stopword.ToLower();
            }
            inputFile.Close();
        }

    }
}